1 Data preparation

1.1 Data wrangling

1.1.1 Load data

### Load data
df_dtw_zoom = read_csv("data/dtw_distance_zoom.csv") %>% 
  rename(target = target_2) %>%
  select(comparison_id, pair, target, average_distance, average_distance_xyz, hands_dtw) %>%
  mutate(dataset = "zoom",
         pair = as.factor(pair),
         target = as.factor(target))

df_sim_coding_zoom = read_csv("data/similarity_coding_zoom.csv") %>% 
  rename(position = location) %>% 
  mutate(total_similarity = handshape + movement + orientation + position,
         ### binary coding
         handshape_bin = ifelse(handshape >= 4, 1, 0),
         movement_bin = ifelse(movement >= 4, 1, 0),
         orientation_bin = ifelse(orientation >= 4, 1, 0),
         position_bin = ifelse(position >= 4, 1, 0),
         n_features = handshape_bin + movement_bin + orientation_bin + position_bin,
         handshape_bin = factor(handshape_bin, 
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         movement_bin = factor(movement_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         orientation_bin = factor(orientation_bin,
                                   levels = c(0, 1), 
                                   labels = c("not similar", "similar")),
         position_bin = factor(position_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         ### z-score
         handshape_z = scale(handshape)[,1],
         movement_z = scale(movement)[,1],
         orientation_z = scale(orientation)[,1],
         position_z = scale(position)[,1],
         total_similarity_z = scale(total_similarity)[,1]) %>% 
  select(-notes)

### combine dtw and coding data for zoom dataset
df_dtw_zoom = right_join(df_dtw_zoom, df_sim_coding_zoom)

### convert to long format for data visualization
df_dtw_zoom_long = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape", "movement", "orientation", "position"), 
               names_to = "feature", 
               values_to = "similarity") %>% 
  select(-ends_with("_z"), -ends_with("_bin"))

df_dtw_zoom_long_bin = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape_bin", "movement_bin", "orientation_bin", "position_bin"), 
               names_to = "feature", 
               values_to = "similarity_bin") %>% 
  mutate(feature = factor(sub("_bin", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_z"))

df_dtw_zoom_long_z = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape_z", "movement_z", "orientation_z", "position_z"), 
               names_to = "feature", 
               values_to = "similarity_z") %>% 
  mutate(feature = factor(sub("_z", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_bin"))

df_dtw_zoom_long = left_join(df_dtw_zoom_long, df_dtw_zoom_long_z) %>% 
  left_join(., df_dtw_zoom_long_bin) %>%
  select(-starts_with("total_similarity"))

## for partial residual plot
df_dtw_zoom$handshape_res = residuals(lm(handshape ~ movement + orientation + position, data = df_dtw_zoom))
df_dtw_zoom$movement_res = residuals(lm(movement ~ handshape + orientation + position, data = df_dtw_zoom))
df_dtw_zoom$orientation_res = residuals(lm(orientation ~ handshape + movement + position, data = df_dtw_zoom))
df_dtw_zoom$position_res = residuals(lm(position ~ handshape + movement + orientation, data = df_dtw_zoom))

df_dtw_zoom_long_res = df_dtw_zoom %>%
  pivot_longer(cols = c("handshape_res", "movement_res", "orientation_res", "position_res"), 
               names_to = "feature", 
               values_to = "similarity_res") %>% 
  mutate(feature = factor(sub("_res", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(!dataset:total_similarity_z)

1.2 Data visualization

1.2.1 correlation for each feature

scp_feature = df_dtw_zoom_long %>%
  ggplot(aes(x = similarity, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 5),
                     breaks = seq(0, 5, 1)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

ggplotly(scp_feature)


1.2.2 partial residual plot

scp_feature_res = df_dtw_zoom_long_res %>%
  ggplot(aes(x = similarity_res, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score (residuals)", 
       y="Normalized DTW distance") +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_res


1.2.3 correlation for total similarity score

scp_total = df_dtw_zoom %>%
  ggplot(aes(x = total_similarity, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Total similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 20),
                     breaks = seq(0, 20, 5)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total, fill = "lightblue")

# ggplotly(scp_total)


1.2.4 [z] correlation for each feature

scp_feature_z = df_dtw_zoom_long %>%
  ggplot(aes(x = similarity_z, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.5, 2),
                     breaks = seq(-2.5, 2, 1)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_z


1.2.5 [z] correlation for total similarity score

scp_total_z = df_dtw_zoom %>%
  ggplot(aes(x = total_similarity_z, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized total similarity score (z)", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.2, 1.5),
                     breaks = seq(-2, 1.5, 0.5)) +
  scale_y_continuous(limits = c(0, 1),
                     breaks = seq(0, 1, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total_z, fill = "lightblue")


1.3 Statistical analysis

1.3.1 Check the normality assumption of errors

Linear mixed-effects models assume that the residuals are normally distributed. Let’s check this assumption by plotting the residuals of the models. For this, we will use the check_model() function from the performance package.

model = lmer(average_distance_xyz ~ 1 + (1|pair) + (1|target), 
             data = subset(df_dtw_zoom))
check_model(model)

model_log = lmer(log(average_distance_xyz) ~ 1 + (1|pair) + (1|target), 
                 data = subset(df_dtw_zoom))
check_model(model_log)

Log-normal model returned a sigular fit (i.e., the model is not able to estimate the variance of the random effects for pair). Therefore, we will use linear regression for the analysis.


1.3.2 Correlation for total similarity

x = df_dtw_zoom$total_similarity
y = df_dtw_zoom$average_distance_xyz

cor.test(x, y, method="pearson")
## 
##  Pearson's product-moment correlation
## 
## data:  x and y
## t = -3, df = 98, p-value = 0.003
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  -0.466 -0.106
## sample estimates:
##    cor 
## -0.296
cor.test(x, y, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  x and y
## S = 226981, p-value = 0.0002
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.362

The correlation between the total similarity score and the normalized DTW distance is significant (r = -0.296, p = 0.004), suggesting that the more similar features two gestures have, the smaller the distance is.


1.3.3 LMM for total similarity score

lmer_zoom_total = lmer(average_distance_xyz ~ 
                           total_similarity_z +
                           (1|pair) + (1|target),
                         data = df_dtw_zoom)

summ(lmer_zoom_total, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -65.149
BIC -52.123
Pseudo-R² (fixed effects) 0.093
Pseudo-R² (total) 0.162
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.349 0.019 18.201 9.257 0.000
total_similarity_z -0.051 0.016 -3.225 93.777 0.002
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.028
target (Intercept) 0.034
Residual 0.152
Grouping Variables
Group # groups ICC
pair 22 0.031
target 16 0.045
check_model(lmer_zoom_total)


1.3.4 Correlation for each feature

print("Handshape")
## [1] "Handshape"
cor.test(df_dtw_zoom$handshape, df_dtw_zoom$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$handshape and df_dtw_zoom$average_distance_xyz
## S = 198505, p-value = 0.06
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.191
print("Orientation")
## [1] "Orientation"
cor.test(df_dtw_zoom$orientation, df_dtw_zoom$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$orientation and df_dtw_zoom$average_distance_xyz
## S = 206904, p-value = 0.02
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.242
print("Movement")
## [1] "Movement"
cor.test(df_dtw_zoom$movement, df_dtw_zoom$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$movement and df_dtw_zoom$average_distance_xyz
## S = 213797, p-value = 0.004
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.283
print("Position")
## [1] "Position"
cor.test(df_dtw_zoom$position, df_dtw_zoom$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom$position and df_dtw_zoom$average_distance_xyz
## S = 250467, p-value = 0.0000001
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.503


1.3.5 LMM for each feature

### each feature
lmer_zoom_shape = lmer(average_distance_xyz ~ 
                 handshape_z + (1|pair) + (1|target),
               data = df_dtw_zoom)
summ(lmer_zoom_shape, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -56.998
BIC -43.972
Pseudo-R² (fixed effects) 0.015
Pseudo-R² (total) 0.060
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.351 0.019 18.821 7.653 0.000
handshape_z -0.020 0.016 -1.238 96.778 0.219
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.020
target (Intercept) 0.029
Residual 0.161
Grouping Variables
Group # groups ICC
pair 22 0.015
target 16 0.031
lmer_zoom_movement = lmer(average_distance_xyz ~ 
                             movement_z + (1|pair) + (1|target),
                           data = df_dtw_zoom)
summ(lmer_zoom_movement, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -61.372
BIC -48.346
Pseudo-R² (fixed effects) 0.058
Pseudo-R² (total) 0.113
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.351 0.019 18.637 8.177 0.000
movement_z -0.040 0.016 -2.481 95.894 0.015
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.022
target (Intercept) 0.032
Residual 0.156
Grouping Variables
Group # groups ICC
pair 22 0.018
target 16 0.041
lmer_zoom_orientation = lmer(average_distance_xyz ~ 
                                orientation_z + (1|pair) + (1|target),
                              data = df_dtw_zoom)
summ(lmer_zoom_orientation, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -59.376
BIC -46.350
Pseudo-R² (fixed effects) 0.039
Pseudo-R² (total) 0.101
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.350 0.019 18.150 8.251 0.000
orientation_z -0.033 0.016 -2.030 94.347 0.045
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.029
target (Intercept) 0.029
Residual 0.158
Grouping Variables
Group # groups ICC
pair 22 0.032
target 16 0.033
lmer_zoom_position = lmer(average_distance_xyz ~ 
                              position_z + (1|pair) + (1|target),
                            data = df_dtw_zoom)
summ(lmer_zoom_position, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -77.154
BIC -64.129
Pseudo-R² (fixed effects) 0.195
Pseudo-R² (total) 0.248
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.349 0.018 19.804 9.665 0.000
position_z -0.073 0.015 -4.944 96.571 0.000
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.018
target (Intercept) 0.033
Residual 0.144
Grouping Variables
Group # groups ICC
pair 22 0.016
target 16 0.050
### all features
lmer_zoom_feature = lmer(average_distance_xyz ~ 
                            handshape_z + movement_z + orientation_z + position_z +
                            (1|pair) + (1|target),
                          data = df_dtw_zoom)

summ(lmer_zoom_feature, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -53.097
BIC -32.255
Pseudo-R² (fixed effects) 0.192
Pseudo-R² (total) 0.244
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.349 0.018 19.525 9.385 0.000
handshape_z -0.001 0.021 -0.025 94.887 0.980
movement_z 0.007 0.021 0.335 94.358 0.738
orientation_z -0.003 0.024 -0.143 92.887 0.887
position_z -0.076 0.019 -4.013 94.250 0.000
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.019
target (Intercept) 0.033
Residual 0.146
Grouping Variables
Group # groups ICC
pair 22 0.016
target 16 0.049
cov2cor(vcov(lmer_zoom_feature))
## 5 x 5 Matrix of class "corMatrix"
##               (Intercept) handshape_z movement_z orientation_z position_z
## (Intercept)       1.00000    -0.00179    -0.0126        0.0102     0.0171
## handshape_z      -0.00179     1.00000    -0.0582       -0.6197     0.0693
## movement_z       -0.01261    -0.05821     1.0000       -0.3121    -0.4659
## orientation_z     0.01016    -0.61973    -0.3121        1.0000    -0.1506
## position_z        0.01705     0.06926    -0.4659       -0.1506     1.0000

Regressions on each feature show a significant negative association between DTW distance and movement, orientation, and position (handshape didn’t reach significance). However, the model with all features as fixed effects shows such significant negative correlation only for position.

To check why the effects “disappeared”, we examined the correlation of fixed effects and found a strong negative correlation between handshape and orientation (-0.6) and between movement and position (-0.47). This suggests that when the slope estimate for orientation is more extremely negative, the slope estimate for handshape becomes flatter (or more positive). This is likely due to the fact that handshape and orientation are both reflected in relative finger tip positions. Similar relationship was found for movement and position, which is likely due to the fact that they are both reflected in the wrist positions.

Again, this suggests a need for developing a different approach to capturing handshape similarity independently of hand orientation.


2 Revised DTW pipeline for Zoom dataset

A qualitative check on incongruent cases where a pair of gestures was annotated as similar but it’s DTW distance was large revealed that the majority of such cases is because of mirrored movements. To tackle this issue, for gestures where speaker A and B used the opposing hands (e.g., speaker used left-hand and speaker B right-hand), we will calculate DTW distance for both original and flipped videos and take the minimum of the two.

2.1 Data wrangling

2.1.1 Load data

### Load data
df_dtw_zoom_v2 = read_csv("data/dtw_distance_zoom_mirrored.csv") %>% 
  rename(target = target_2) %>%
  select(comparison_id, pair, target, average_distance, average_distance_xyz, hands_dtw) %>%
  mutate(dataset = "zoom",
         pair = as.factor(pair),
         target = as.factor(target))

df_sim_coding_zoom = read_csv("data/similarity_coding_zoom.csv") %>% 
  rename(position = location) %>% 
  mutate(total_similarity = handshape + movement + orientation + position,
         ### binary coding
         handshape_bin = ifelse(handshape >= 4, 1, 0),
         movement_bin = ifelse(movement >= 4, 1, 0),
         orientation_bin = ifelse(orientation >= 4, 1, 0),
         position_bin = ifelse(position >= 4, 1, 0),
         n_features = handshape_bin + movement_bin + orientation_bin + position_bin,
         handshape_bin = factor(handshape_bin, 
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         movement_bin = factor(movement_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         orientation_bin = factor(orientation_bin,
                                   levels = c(0, 1), 
                                   labels = c("not similar", "similar")),
         position_bin = factor(position_bin,
                                levels = c(0, 1), 
                                labels = c("not similar", "similar")),
         ### z-score
         handshape_z = scale(handshape)[,1],
         movement_z = scale(movement)[,1],
         orientation_z = scale(orientation)[,1],
         position_z = scale(position)[,1],
         total_similarity_z = scale(total_similarity)[,1]) %>% 
  select(-notes)

### combine dtw and coding data for zoom dataset
df_dtw_zoom_v2 = right_join(df_dtw_zoom_v2, df_sim_coding_zoom)

### convert to long format for data visualization
df_dtw_zoom_v2_long = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape", "movement", "orientation", "position"), 
               names_to = "feature", 
               values_to = "similarity") %>% 
  select(-ends_with("_z"), -ends_with("_bin"))

df_dtw_zoom_v2_long_bin = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape_bin", "movement_bin", "orientation_bin", "position_bin"), 
               names_to = "feature", 
               values_to = "similarity_bin") %>% 
  mutate(feature = factor(sub("_bin", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_z"))

df_dtw_zoom_v2_long_z = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape_z", "movement_z", "orientation_z", "position_z"), 
               names_to = "feature", 
               values_to = "similarity_z") %>% 
  mutate(feature = factor(sub("_z", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(-handshape, -movement, -orientation, -position, -ends_with("_bin"))

df_dtw_zoom_v2_long = left_join(df_dtw_zoom_v2_long, df_dtw_zoom_v2_long_z) %>% 
  left_join(., df_dtw_zoom_v2_long_bin) %>%
  select(-starts_with("total_similarity"))

## for partial residual plot
df_dtw_zoom_v2$handshape_res = residuals(lm(handshape ~ movement + orientation + position, data = df_dtw_zoom_v2))
df_dtw_zoom_v2$movement_res = residuals(lm(movement ~ handshape + orientation + position, data = df_dtw_zoom_v2))
df_dtw_zoom_v2$orientation_res = residuals(lm(orientation ~ handshape + movement + position, data = df_dtw_zoom_v2))
df_dtw_zoom_v2$position_res = residuals(lm(position ~ handshape + movement + orientation, data = df_dtw_zoom_v2))

df_dtw_zoom_v2_long_res = df_dtw_zoom_v2 %>%
  pivot_longer(cols = c("handshape_res", "movement_res", "orientation_res", "position_res"), 
               names_to = "feature", 
               values_to = "similarity_res") %>% 
  mutate(feature = factor(sub("_res", "", feature),
                          levels = c("handshape", "movement", "orientation", "position"))) %>% 
  select(!dataset:total_similarity_z)


2.2 Data visualization

2.2.1 correlation for each feature

scp_feature = df_dtw_zoom_v2_long %>%
  ggplot(aes(x = similarity, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 5),
                     breaks = seq(0, 5, 1)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

ggplotly(scp_feature)


2.2.2 partial residual plot

scp_feature_res = df_dtw_zoom_v2_long_res %>%
  ggplot(aes(x = similarity_res, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Similarity score (residuals)", 
       y="Normalized DTW distance") +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_res


2.2.3 correlation for total similarity score

scp_total = df_dtw_zoom_v2 %>%
  ggplot(aes(x = total_similarity, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Total similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(0, 20),
                     breaks = seq(0, 20, 5)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total, fill = "lightblue")

# ggplotly(scp_total)


2.2.4 [z] correlation for each feature

scp_feature_z = df_dtw_zoom_v2_long %>%
  ggplot(aes(x = similarity_z, y = average_distance_xyz)) +
  geom_point(size = 0.8, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized similarity score", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.5, 2),
                     breaks = seq(-2.5, 2, 1)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines")) +
  facet_wrap(~feature, nrow = 2)

scp_feature_z


2.2.5 [z] correlation for total similarity score

scp_total_z = df_dtw_zoom_v2 %>%
  ggplot(aes(x = total_similarity_z, y = average_distance_xyz)) +
  geom_point(size = 1, alpha = 0.7) +
  geom_smooth(method = "lm") +
  labs(x="Standardized total similarity score (z)", 
       y="Normalized DTW distance") +
  scale_x_continuous(limits = c(-2.2, 1.5),
                     breaks = seq(-2, 1.5, 0.5)) +
  scale_y_continuous(limits = c(0, 0.8),
                     breaks = seq(0, 0.8, 0.2)) +
  theme_classic(base_size = 14) +
  theme(axis.text.x = element_text(colour = "black", size = 13),
        axis.text.y = element_text(colour = "black", size = 13),
        axis.title = element_text(face = 'bold'),
        axis.title.x = element_text(vjust = -2),
        axis.title.y = element_text(vjust = 2),
        strip.text = element_text(face = 'bold'),
        legend.position = "none",
        plot.margin = unit(c(1.1,1.1,1.1,1.1), "lines"))

ggExtra::ggMarginal(scp_total_z, fill = "lightblue")


2.3 Statistical analysis

2.3.1 Check the normality assumption of errors

Linear mixed-effects models assume that the residuals are normally distributed. Let’s check this assumption by plotting the residuals of the models. For this, we will use the check_model() function from the performance package.

model = lmer(average_distance_xyz ~ 1 + (1|pair), 
             data = subset(df_dtw_zoom_v2))
check_model(model)

model_log = lmer(log(average_distance_xyz) ~ 1 + (1|pair), 
                 data = subset(df_dtw_zoom_v2))
check_model(model_log)

Models including random intercepts for dyads and items returned a singular fit, most likely because of the small sample size. After removing the by-item random intercept, the model converged only for the linear model. Therefore, we will use linear regression for the analysis.


2.3.2 Correlation for total similarity

x = df_dtw_zoom_v2$total_similarity
y = df_dtw_zoom_v2$average_distance_xyz

cor.test(x, y, method="pearson")
## 
##  Pearson's product-moment correlation
## 
## data:  x and y
## t = -6, df = 98, p-value = 0.00000002
## alternative hypothesis: true correlation is not equal to 0
## 95 percent confidence interval:
##  -0.654 -0.366
## sample estimates:
##    cor 
## -0.525
cor.test(x, y, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  x and y
## S = 258048, p-value = 0.000000003
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.548

The correlation between the total similarity score and the normalized DTW distance is significant (r = -0.296, p = 0.004), suggesting that the more similar features two gestures have, the smaller the distance is.


2.3.3 LMM for total similarity score

lmer_zoom_total = lmer(average_distance_xyz ~ 
                           total_similarity_z +
                           (1|pair),
                         data = df_dtw_zoom_v2)

summ(lmer_zoom_total, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -133.125
BIC -122.704
Pseudo-R² (fixed effects) 0.278
Pseudo-R² (total) 0.328
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.309 0.013 23.038 14.433 0.000
total_similarity_z -0.070 0.011 -6.291 93.911 0.000
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.030
Residual 0.109
Grouping Variables
Group # groups ICC
pair 22 0.068
check_model(lmer_zoom_total)


2.3.4 Correlation for each feature

print("Handshape")
## [1] "Handshape"
cor.test(df_dtw_zoom_v2$handshape, df_dtw_zoom_v2$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$handshape and df_dtw_zoom_v2$average_distance_xyz
## S = 227495, p-value = 0.0002
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.365
print("Orientation")
## [1] "Orientation"
cor.test(df_dtw_zoom_v2$orientation, df_dtw_zoom_v2$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$orientation and df_dtw_zoom_v2$average_distance_xyz
## S = 223012, p-value = 0.0006
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.338
print("Movement")
## [1] "Movement"
cor.test(df_dtw_zoom_v2$movement, df_dtw_zoom_v2$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$movement and df_dtw_zoom_v2$average_distance_xyz
## S = 233120, p-value = 0.00004
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.399
print("Position")
## [1] "Position"
cor.test(df_dtw_zoom_v2$position, df_dtw_zoom_v2$average_distance_xyz, method="spearman")
## 
##  Spearman's rank correlation rho
## 
## data:  df_dtw_zoom_v2$position and df_dtw_zoom_v2$average_distance_xyz
## S = 281737, p-value = 0.000000000000002
## alternative hypothesis: true rho is not equal to 0
## sample estimates:
##    rho 
## -0.691


2.3.5 LMM for each feature

### each feature
lmer_zoom_shape = lmer(average_distance_xyz ~ 
                 handshape_z + (1|pair),
               data = df_dtw_zoom_v2)
summ(lmer_zoom_shape, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -111.445
BIC -101.025
Pseudo-R² (fixed effects) 0.105
Pseudo-R² (total) 0.139
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.309 0.014 22.090 12.438 0.000
handshape_z -0.043 0.013 -3.429 96.599 0.001
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.025
Residual 0.123
Grouping Variables
Group # groups ICC
pair 22 0.038
lmer_zoom_movement = lmer(average_distance_xyz ~ 
                             movement_z + (1|pair),
                           data = df_dtw_zoom_v2)
summ(lmer_zoom_movement, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -118.297
BIC -107.876
Pseudo-R² (fixed effects) 0.164
Pseudo-R² (total) 0.196
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.310 0.014 22.974 13.656 0.000
movement_z -0.054 0.012 -4.438 96.736 0.000
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.024
Residual 0.119
Grouping Variables
Group # groups ICC
pair 22 0.038
lmer_zoom_orientation = lmer(average_distance_xyz ~ 
                                orientation_z + (1|pair),
                              data = df_dtw_zoom_v2)
summ(lmer_zoom_orientation, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -112.806
BIC -102.385
Pseudo-R² (fixed effects) 0.120
Pseudo-R² (total) 0.182
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.309 0.015 20.676 13.725 0.000
orientation_z -0.046 0.012 -3.723 95.334 0.000
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.033
Residual 0.121
Grouping Variables
Group # groups ICC
pair 22 0.071
lmer_zoom_position = lmer(average_distance_xyz ~ 
                              position_z + (1|pair),
                            data = df_dtw_zoom_v2)
summ(lmer_zoom_position, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -157.211
BIC -146.790
Pseudo-R² (fixed effects) 0.438
Pseudo-R² (total) 0.462
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.309 0.011 27.471 15.242 0.000
position_z -0.088 0.010 -8.785 97.982 0.000
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.021
Residual 0.098
Grouping Variables
Group # groups ICC
pair 22 0.044
### all features
lmer_zoom_feature = lmer(average_distance_xyz ~ 
                            handshape_z + movement_z + orientation_z + position_z +
                            (1|pair),
                          data = df_dtw_zoom_v2)

summ(lmer_zoom_feature, digits = 3)
Observations 100
Dependent variable average_distance_xyz
Type Mixed effects linear regression
AIC -134.496
BIC -116.260
Pseudo-R² (fixed effects) 0.451
Pseudo-R² (total) 0.475
Fixed Effects
Est. S.E. t val. d.f. p
(Intercept) 0.308 0.011 27.664 15.178 0.000
handshape_z -0.025 0.014 -1.802 94.699 0.075
movement_z 0.002 0.014 0.128 94.975 0.899
orientation_z 0.008 0.016 0.519 94.910 0.605
position_z -0.085 0.012 -6.859 94.981 0.000
p values calculated using Satterthwaite d.f.
Random Effects
Group Parameter Std. Dev.
pair (Intercept) 0.020
Residual 0.097
Grouping Variables
Group # groups ICC
pair 22 0.042
cov2cor(vcov(lmer_zoom_feature))
## 5 x 5 Matrix of class "corMatrix"
##               (Intercept) handshape_z movement_z orientation_z position_z
## (Intercept)       1.00000     0.00177    -0.0252        0.0265     0.0148
## handshape_z       0.00177     1.00000    -0.0559       -0.6135     0.0481
## movement_z       -0.02519    -0.05590     1.0000       -0.3260    -0.4531
## orientation_z     0.02645    -0.61347    -0.3260        1.0000    -0.1393
## position_z        0.01483     0.04805    -0.4531       -0.1393     1.0000

Regressions on each feature show a significant negative association between DTW distance for all features. However, the model with all features as fixed effects shows such significant negative correlation only for position.

To check why the effects “disappeared”, we examined the correlation of fixed effects and found a strong negative correlation between handshape and orientation (-0.6) and between movement and position (-0.45). This suggests that when the slope estimate for orientation is more extremely negative, the slope estimate for handshape becomes flatter (or more positive). This is likely due to the fact that handshape and orientation are both reflected in relative finger tip positions. Similar relationship was found for movement and position, which is likely due to the fact that they are both reflected in the wrist positions.


3 Session Info

sessionInfo()
## R version 4.4.0 (2024-04-24 ucrt)
## Platform: x86_64-w64-mingw32/x64
## Running under: Windows 11 x64 (build 22631)
## 
## Matrix products: default
## 
## 
## locale:
## [1] LC_COLLATE=English_United States.utf8 
## [2] LC_CTYPE=English_United States.utf8   
## [3] LC_MONETARY=English_United States.utf8
## [4] LC_NUMERIC=C                          
## [5] LC_TIME=English_United States.utf8    
## 
## time zone: Europe/Amsterdam
## tzcode source: internal
## 
## attached base packages:
## [1] stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] kableExtra_1.4.0   sjPlot_2.8.16      performance_0.11.0 glmmTMB_1.1.9     
##  [5] jtools_2.2.2       lmerTest_3.1-3     lme4_1.1-35.3      Matrix_1.7-0      
##  [9] plotrix_3.8-4      patchwork_1.2.0    plotly_4.10.4      ggtext_0.1.2      
## [13] lubridate_1.9.3    forcats_1.0.0      stringr_1.5.1      dplyr_1.1.4       
## [17] purrr_1.0.2        readr_2.1.5        tidyr_1.3.1        tibble_3.2.1      
## [21] ggplot2_3.5.1      tidyverse_2.0.0   
## 
## loaded via a namespace (and not attached):
##  [1] rlang_1.1.4         magrittr_2.0.3      compiler_4.4.0     
##  [4] mgcv_1.9-1          systemfonts_1.1.0   vctrs_0.6.5        
##  [7] pkgconfig_2.0.3     crayon_1.5.2        fastmap_1.2.0      
## [10] labeling_0.4.3      pander_0.6.5        utf8_1.2.4         
## [13] promises_1.3.0      rmarkdown_2.27      tzdb_0.4.0         
## [16] nloptr_2.0.3        bit_4.0.5           xfun_0.44          
## [19] cachem_1.1.0        jsonlite_2.0.0      highr_0.11         
## [22] later_1.3.2         sjmisc_2.8.10       ggeffects_1.6.0    
## [25] parallel_4.4.0      R6_2.5.1            bslib_0.7.0        
## [28] stringi_1.8.4       boot_1.3-30         jquerylib_0.1.4    
## [31] numDeriv_2016.8-1.1 estimability_1.5.1  Rcpp_1.0.13        
## [34] knitr_1.46          httpuv_1.6.15       splines_4.4.0      
## [37] timechange_0.3.0    tidyselect_1.2.1    rstudioapi_0.16.0  
## [40] yaml_2.3.8          TMB_1.9.11          miniUI_0.1.1.1     
## [43] sjlabelled_1.2.0    lattice_0.22-6      bayestestR_0.13.2  
## [46] shiny_1.8.1.1       withr_3.0.1         coda_0.19-4.1      
## [49] evaluate_0.23       xml2_1.3.6          pillar_1.9.0       
## [52] insight_0.19.11     generics_0.1.3      vroom_1.6.5        
## [55] hms_1.1.3           munsell_0.5.1       scales_1.3.0       
## [58] minqa_1.2.7         xtable_1.8-4        glue_1.8.0         
## [61] emmeans_1.10.3      lazyeval_0.2.2      tools_4.4.0        
## [64] see_0.8.4           data.table_1.16.2   mvtnorm_1.3-1      
## [67] grid_4.4.0          crosstalk_1.2.1     datawizard_0.10.0  
## [70] colorspace_2.1-1    nlme_3.1-164        cli_3.6.3          
## [73] fansi_1.0.6         viridisLite_0.4.2   svglite_2.1.3      
## [76] sjstats_0.19.0      gtable_0.3.5        sass_0.4.9         
## [79] digest_0.6.37       ggrepel_0.9.6       htmlwidgets_1.6.4  
## [82] farver_2.1.2        htmltools_0.5.8.1   lifecycle_1.0.4    
## [85] httr_1.4.7          mime_0.12           gridtext_0.1.5     
## [88] ggExtra_0.10.1      bit64_4.0.5         MASS_7.3-60.2